package main

import (
	"bufio"
	"bytes"
	"code.google.com/p/go.text/encoding/simplifiedchinese"
	"code.google.com/p/go.text/transform"
	"flag"
	"fmt"
	goq "github.com/PuerkitoBio/goquery"
	"io/ioutil"
	"net"
	"net/http"
	"os"
	//"runtime"
	//"strconv"
	"strings"
	"sync"
	"time"
)

var (
	main_domain = "" //主域名
	waitgroup   = &sync.WaitGroup{}
	LOG         = ""
	ErrLOG      = ""
)

func main() {
	var citycode = flag.String("citycode", "bj", "请输入城市编号") //城市编号
	flag.Parse()
	main_domain = "http://" + *citycode + ".esf.sina.com.cn"
	LOG = *citycode + ".esf.sina.com.cn"
	ErrLOG = *citycode + ".esf.sina.com.cn"

	//读取要抓取的列表
	_, e := os.Stat("./cfg.ini")
	if e != nil {
		s("./cfg.ini 文件不存在！")
		os.Exit(-1)
	}
	f, e := os.OpenFile("./cfg.ini", os.O_RDONLY, 0777)
	defer f.Close()
	buf := bufio.NewReader(f)
	for {
		line, _, e := buf.ReadLine()
		if e != nil {
			break
		}
		tmparr := strings.Split(string(line[:]), ",")
		houseid := tmparr[0]      //房源ID
		unitprice := tmparr[1]    //成交单价
		saleprice := tmparr[2]    //成交总价
		solddate := tmparr[3]     //成交日期
		soldarr := strings.Split(solddate, "-")
		sold_y := soldarr[0]
		sold_m := soldarr[1]
		sold_d := soldarr[2]

		houseinfo := getHouseInfo(houseid)
		if houseinfo == nil {
			errlog(houseid + " 查找不到")
			continue
		}
		str := houseinfo["district"]          //城区
		str += "|" + houseinfo["block"]       //板块
		str += "|" + houseinfo["community"]   //小区名称
		str += "|" + houseinfo["houseid"]     //房源ID
		str += "|" + unitprice                //成交单价
		str += "|" + saleprice                //成交总价
		str += "|" + "--"                     //佣金
		str += "|" + houseinfo["pptype"]      //房屋性质
		str += "|" + houseinfo["buildarea"]   //面积
		str += "|" + houseinfo["room"]        //居
		str += "|" + houseinfo["direction"]   //朝向
		str += "|" + houseinfo["floor"]       //所在层高
		str += "|" + houseinfo["total_floor"] //总层高
		str += "|" + sold_y                   //成交年
		str += "|" + sold_m                   //成交月
		str += "|" + sold_d                   //成交日
		str += "|" + houseinfo["companyname"] //经纪公司
		str += "|" + houseinfo["agentname"]   //经纪人
		str += "|" + houseinfo["mobile"]      //联系电话
		str += "|" + "--"                     //销售
		str += "|" + "--"                     //部门

		outlog(str)
	}
	fmt.Println("done.")
}

//获取房源信息
func getHouseInfo(houseid string) map[string]string {
	tmpurl := fmt.Sprintf("%s/detail/%s", main_domain, houseid)
	doc, e := getDoc(tmpurl)
	if e != nil {
		return nil
	}
	houseinfo := make(map[string]string)
	
	tmp := doc.Find(".nav-crumbs a")
	if tmp.Length() < 5 {
		return nil
	}
	
	houseinfo["district"] = tmp.Eq(2).Text()
	houseinfo["block"] = tmp.Eq(3).Text()
	houseinfo["community"] = tmp.Eq(4).Text()
	houseinfo["houseid"] = houseid
	
	houseinfo["pptype"] = doc.Find("td > span:contains('类型：')").Parent().Text()
	houseinfo["pptype"] = strings.Replace(houseinfo["pptype"], "类型：", "", -1)
	
	houseinfo["buildarea"] = doc.Find("td > span:contains('面积：')").Parent().Text()
	houseinfo["buildarea"] = strings.Replace(houseinfo["buildarea"], "面积：", "", -1)
	
	houseinfo["room"] = doc.Find("td > span:contains('户型：')").Parent().Text()
	houseinfo["room"] = strings.Replace(houseinfo["room"], "户型：", "", -1)
	
	houseinfo["floor"] = doc.Find("td > span:contains('楼层：')").Parent().Text()
	houseinfo["floor"] = strings.Replace(houseinfo["floor"], "楼层：", "", -1)
	houseinfo["floor"] = strings.Replace(houseinfo["floor"], "层", "", -1)
	tmpfloor := strings.Split(houseinfo["floor"],"/")
	houseinfo["floor"] = tmpfloor[0]
	houseinfo["total_floor"] = tmpfloor[1]
		
	houseinfo["direction"] = doc.Find("td > span:contains('朝向：')").Parent().Text()
	houseinfo["direction"] = strings.Replace(houseinfo["room"], "朝向：", "", -1)
	
	houseinfo["companyname"] = doc.Find(".sblue").Eq(0).Text()
	houseinfo["agentname"] = doc.Find(".i-right dl dd a").Text()
	houseinfo["mobile"] = filterStr(doc.Find(".h-tel").Text())

	return houseinfo
}

//获取doc
func getDoc(url string) (*goq.Document, error) {
	res, e := getResp(url)
	if e != nil {
		return nil, e
	}
	doc, e := goq.NewDocumentFromResponse(res)
	if e != nil {
		return nil, e
	}
	return doc, nil
}

//获取response
func getResp(url string) (*http.Response, error) {
	req, e := http.NewRequest("GET", url, nil)
	req.Header.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8")
	// req.Header.Add("Accept-Encoding", "gzip,deflate,sdch")
	req.Header.Add("Accept-Language", "zh-CN,zh;q=0.8")
	req.Header.Add("Cache-Control", "max-age=0")
	req.Header.Add("Proxy-Connection", "keep-alive")
	req.Header.Add("User-Agent", "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.63 Safari/537.36")
	c := &http.Client{
		Transport: &http.Transport{
			Dial: func(netw, addr string) (net.Conn, error) {
				deadline := time.Now().Add(25 * time.Second)
				c, err := net.DialTimeout(netw, addr, time.Second*20)
				if err != nil {
					return nil, err
				}
				c.SetDeadline(deadline)
				return c, nil
			},
		},
	}
	resp, e := c.Do(req)
	//defer resp.Body.Close()
	if e != nil {
		return nil, e
	}
	return resp, nil
}

//gbk->utf8
func G2U(s string) string {
	ret, e := gbktoutf8([]byte(s))
	if e != nil {
		return ""
	}
	return string(ret)
}

//utf8->gbk
func U2G(s string) string {
	ret, e := utf8togbk([]byte(s))
	if e != nil {
		return ""
	}
	return string(ret)
}

func gbktoutf8(s []byte) ([]byte, error) {
	reader := transform.NewReader(bytes.NewReader(s), simplifiedchinese.GBK.NewDecoder())
	d, e := ioutil.ReadAll(reader)
	if e != nil {
		return nil, e
	}
	return d, nil
}
func utf8togbk(s []byte) ([]byte, error) {
	reader := transform.NewReader(bytes.NewReader(s), simplifiedchinese.GBK.NewEncoder())
	d, e := ioutil.ReadAll(reader)
	if e != nil {
		return nil, e
	}
	return d, nil
}

func filterStr(str string) string {
	str = strings.Replace(str, "\r\n", "", -1)
	str = strings.Replace(str, "\r", "", -1)
	str = strings.Replace(str, "\n", "", -1)
	str = strings.Replace(str, "|", "", -1)
	str = strings.TrimSpace(str)
	return str
}

func s(a ...interface{}) {
	fmt.Println(a)
}

//outlog
func outlog(str string) {
	pwd, _ := os.Getwd()
	outlog := fmt.Sprintf("%s/%s_log.txt", pwd, LOG)
	file, err := os.OpenFile(outlog, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600)
	defer file.Close()
	if err != nil {
		return
	}
	str += "\r\n"
	file.WriteString(str)
}

//errlog
func errlog(str string) {
	pwd, _ := os.Getwd()
	outlog := fmt.Sprintf("%s/%s_err.txt", pwd, ErrLOG)
	file, err := os.OpenFile(outlog, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600)
	defer file.Close()
	if err != nil {
		return
	}
	str += "\r\n"
	file.WriteString(str)
}
